<?php
/**
 * Created by PhpStorm.
 * User: vbao
 * Date: 2016/6/13
 * Time: 10:55
 */

namespace Home\Controller;
use Think\Controller;
use QL\QueryList;

class CaijiController extends Controller
{

    public function _initialize()
    {
        set_time_limit(0);
        import('Org.JAE.QueryList');
        header("Content-type: text/html; charset=utf-8");

    }

    //public SLEEP_TIME; //延迟时间

    /**
     * 采集文章网址并放入待采集的数据库
     *
     */

    public function getArticleUrlList(){



        //if(IS_POST){

            //实时显示提示信息 html
            $ii = 0;
            G('begin');
            showMsgHtml();
            showMsg("准备采集文章链接!");
            sleep(C(SLEEP_TIME));



            $listType = I('get.type', 2);  //采集类型 1单一网址 ， 2通配符网址， 3json， 4rss


            $aaa = [
                'caijiurl' => ['.article-title a','href'],
            ];

            $aaa = serialize($aaa);

        $cjListRole = [
            'caijiurl' => ['.excerpt-content h3 a', 'href'],
        ];
        $cjArticleRole = [
            'title' => ['.content h1', 'text'],
            'content' => ['.content_body', 'text', ''], // -.edit-credits .newsletter .ctx-module-container
            'cover' => ['.featured-image img', 'src'],
        ];

        $caijiR = serialize($cjArticleRole);

//        dump($caijiR);die;

            switch ($listType)
            {
                case 1:
                    // 单一网址采集 传入参数 1. cid page  2. listURL
                    $page = I('get.page', 1);
                    $cid = I('get.cid', 601);

                    if($cid){
                        $cateInfo = M('Category')->find($cid);

                        $cateUrl = $cateInfo['url'];
                        $listURL = $cateUrl.$page.'/';
                    }else{
                        $listURL = I('get.listURL');
                    }
                    $cj_id = I('get.cj_id', 8);
                    $listRole = get_list_role($cj_id);
                    $articleListURL = QueryList::Query($listURL, $listRole)->data;

                    break;
                case 2:
                    //
                    $ii = 0;
                    $cData = M('Category')->where('id = 601')->order('id ASC')->select();
//                    $cData = M('Category')->where('status = 1')->order('id ASC')->select();
//                    dump($cData);die;
                    foreach($cData as $cK=>$cV){
                        showMsg(" --  开始采集第".$cV['id']."个分类URL<br />");
                        sleep(C(SLEEP_TIME));

                        $page = F('caiji_cid_'.$cV['id'])?F('caiji_cid_'.$cV['id']):1;
                        
                        //如果页码大于15了，将页码初始化 从第一页开始采集
                        if($page>15){
                            $page = 1;
                        }

                        if($cV['is_ajax']){
                            //menshealth.com
                            if(get_domain($cV['url']) == 'menshealth.com'){

                                $url = $cV['url'].$page.'/11';
                                $post_data = [
                                    'page' => $page,
                                ];
                            }

                            $output = request_post($url,$post_data);
                            $listUrl = $output;

                        }else{
                            $listUrl = $cV['url'].$page.'/';
                        }


                        $listRole = get_list_role($cV['cj_id']);

                        $articleListURL = QueryList::Query($listUrl, $listRole)->data;

//                        dump($articleListURL);die;
                        // 把数据存入数据库
                        $articleCount = 0;
                        foreach ($articleListURL as $url){


                            //获取文章
                            $cjUrl = link_with_domain($cV['cj_id'], $url['caijiurl']);
                            $findArr =  D('Article')->getByCaijiurl($cjUrl);


                            if($cjUrl != $findArr['caijiurl']){

                                //存入文章
//                                $articleCount++;
                                $articleCount ++;
                                $ii++;
                                $url['caijiurl'] = link_with_domain($cV['cj_id'], $url['caijiurl']);
                                $url['status'] = -10;
                                $url['cid'] = $cV['id']?$cV['id']:-10;
                                $url['site_id'] = $cV['site_id']?$cV['site_id']:-10;

                                $url['cj_id'] = $cV['cj_id'];

                                $res = D('Article')->add($url);

                                if($res){
                                    //$articleCount ++;
                                    showMsg($url['caijiurl']." --  添加成功<br />","red", $ii);
                                    sleep(C(SLEEP_TIME));


                                }else{
                                    echo $url['caijiurl']." --  没有加入数据库<br />";
                                }
                            }

                            if($articleCount > 0){
                                //将页码写入缓存
                                F('caiji_cid_'.$cV['id'], $page);
                                break;
                            }
                        }
                        if($articleCount == 0){
                            //将页码写入缓存
                            $fpage = $page+1;
                            F('caiji_cid_'.$cV['id'], $fpage);
                            continue;
                        }
                        showMsg('分类ID = '.$cV['id'].' - 页码 = '.$page.'<br/>','red');
                    }
                    break;
                default:
                    $this->error('type 参数错误');
                    continue;
            }


            showMsg("文章链接采集完成!");
            G('end');
            showMsg("文章链接采集完成! 采集到".$ii."个链接, 消耗了".G('begin','end').'s');
            sleep(C(SLEEP_TIME));

        //}else{
            //获取采集节点规则列表
            $cjRoleList = D('CaijiRole')->where('status = 1')->select();
            $this->assign('cjRoleList', $cjRoleList);

            //获取采集分类列表
            $cjCateList = D('Category')->order('site_id ASC')->select();
            $this->assign('cjCateList', $cjCateList);


            //$this->display();
       //}
    }

    public function caiji(){
        for($i = 1;$i <= 14; $i++){
            //实时显示提示信息 html
            showMsgHtml();
            showMsg("准备采集第个".$i."分类文章!");
            $this->caijiOne($i);
        }
    }

    public function caijiOne($cid = 601){

        $status = 0;  //是否需要翻译
        $limit = 1;
        if($cid){
            $catInfo = M('Category')->find($cid);
            $cjMap['cid'] = $cid;
            $cjMap['site_id'] = $catInfo['site_id'];
        }else{
            $this->error('采集分类ID cid 必须！');
        }

        if($catInfo['site_id'] == 1){
            $author = '美通社';
            $author_url = 'http://www.jiiaa.top/';
        }elseif($catInfo['site_id'] == 2){
            $author = '世界邮报';
            $author_url = 'http://www.viiaa.top/';
        }else{
            $author = '唯宝网';
            $author_url = 'http://www.vbao100.com/';
        }

        $cjMap['status'] = -10;

/*        $cid = I('post.cid');
        $site_id = I('post.site_id');
        $author = I('post.author');
        $author_url = I('post.author_url');
        $status = I('post.status');  //是否需要翻译
        $limit = I('post.limit');*/
        $ii = 0;
        showMsg("开始采集文章!");
        G('begin');
        sleep(SLEEP_TIME);
        //if(IS_POST){
            //从 caiji_articaleUrl 获得文章地址
            $URLs = D('Article')->where($cjMap)->order('id ASC')->limit($limit)->select();
            if(!is_array($URLs)){
                $this->error('没有可以采集的文章了');
            }
            foreach ($URLs as $key => $url){
                //获取规则
                $ArticleRole = get_article_role($url['cj_id']);

                //利用php内置函数 get_meta_tags 获取文章seo_keywords seo_description　内容
                //$seoInfo = $this->curl_getkd($url['caijiurl']);
                $seoInfo = get_keywords_description($url['caijiurl']);

                $cjRes = QueryList::Query($url['caijiurl'], $ArticleRole)->data;
                $article = $cjRes;

                dump($article);die;
                //将数据存入数据库
                $artData = [
                    'id'        =>  $url['id'],
                    'site_id'   =>  $catInfo['site_id'],
                    'cid'       =>  $cid,
                    'title'     =>  $article[0]['title'],
                    'seo_keyword' => $seoInfo['keywords'],
                    'seo_description' => $seoInfo['description'],
                    'author'    =>  $author,
                    'author_url'=>  $author_url,
                    'cover'     =>  $article[0]['cover'],
                    'content'   =>  del3br($article[0]['content']),
                    'status'    =>  $status,
                ];
                $res = D('Article')->save($artData);
                if($article[0]['title']=='' || $article[0]['content']==''){
                    $nullStatus = [
                        'status' => -100,
                    ];
                    D('Article')->where('id = '.$url['id'])->save($nullStatus);
                }
                $ii ++;
                if($res){
                    showMsg($url['caijiurl'].'-- 采集成功',"red",$ii);
                    sleep(SLEEP_TIME);

                }else{
                    showMsg($url['caijiurl'].'-- 采集失败',"",$ii);
                    sleep(SLEEP_TIME);
                }
            }
            G('end');
            showMsg("文章采集完成! 采集到".$ii."个文章, 消耗了".G('begin','end').'s');
            sleep(SLEEP_TIME);
/*        }else{

            $this->display();
        }*/
    }

//利用搜索引擎采集文章
//    public function getList($keyword = '',$web = ''){
//        $keyArr = ['baby','usa','football','boy'];
//        $rand = mt_rand(0,count($keyArr));
//        $keyword = $keyArr[$rand];
//        $web = 'http://www.usatoday.com/story/news';
//        $begin = F('yahoo-begin-'.$keyword) ? intval(F('yahoo-begin-'.$keyword))+10 : 1;
//        $url = 'https://sg.search.yahoo.com/search?p='.$keyword.'&ei=UTF-8&vs='.$web.'&b='.$begin.'&pz=10';
////        echo $url;die; https://sg.search.yahoo.com/search?p=baby&fr2=sb-top-sg.search&fr=sfp&vf=all&vs=http%3A%2F%2Fwww.usatoday.com%2Fstory%2Fnews%2F
//        $rules = [
//            'link'=>['#web h3 a','href'],
//        ];
//
//        //$ql =  new \QueryList($url, $rules);
//        $ql = QueryList::Query($url, $rules)->data;
//        $articleListURL = $ql->jsonArr;
//       /* dump($ql);die;*/
//        foreach($ql as $q){
//            $hasId = preg_match('/(\d{9}).html/',$q['link'],$id);
//            if($hasId){
//                $cnt = M('yahoo_ids')->where(['id'=>$id[1]])->count();
//                if(!$cnt){
//                    M('yahoo_ids')->add(['id'=>$id[1],'status'=>0,'link'=>$q['link']]);
//                }
//            }
//        }
//        F('yahoo-begin-'.$keyword,$begin);
//        dump($ql);
//    }


    /**
     * curl 定时采集 统一调用入口
     */
    public function reset(){


        A('NewCreate')->makeAllHtml(2); //自定义生成html数量
        A('NewCreate')->transArticle(50); //自定义翻译数量 每个1小时翻译一次 每小时翻译次数超过1000条就会被禁用一小时。
//        $this->getArticleUrlList();
//        $this->caiji();
//        A('Create')->transArticle();
//        A('Create')->makeHtml();
    }



}